{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c52efaba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "输入语句：I learn natural language processing with dongshouxueNLP, too.\n",
      "分词结果：['I', 'learn', 'natural', 'language', 'processing', 'with', 'dongshouxueNLP,', 'too.']\n"
     ]
    }
   ],
   "source": [
    "sentence = \"I learn natural language processing with dongshouxueNLP, too.\"\n",
    "tokens = sentence.split(' ')\n",
    "print(f'输入语句：{sentence}')\n",
    "print(f\"分词结果：{tokens}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "53f356c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "输入语句：I learn natural language processing with dongshouxueNLP, too.\n",
      "分词结果：['I', 'learn', 'natural', 'language', 'processing', 'with', 'dongshouxueNLP', 'too']\n"
     ]
    }
   ],
   "source": [
    "#引入正则表达式包\n",
    "import re\n",
    "sentence = \"I learn natural language processing with dongshouxueNLP, too.\"\n",
    "print(f'输入语句：{sentence}')\n",
    "\n",
    "#去除句子中的“,”和“.”\n",
    "sentence = re.sub(r'\\,|\\.','',sentence)\n",
    "tokens = sentence.split(' ')\n",
    "print(f\"分词结果：{tokens}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1a8b7d2d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Did', 'you', 'spend', '3', '4', 'on', 'arxiv', 'org', 'for', 'your', 'pre', 'print', 'No', 'it', 's', 'free', 'It', 's']\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "sentence = \"Did you spend $3.4 on arxiv.org for your pre-print?\"+\\\n",
    "    \" No, it's free! It's ...\"\n",
    "# 其中，\\w表示匹配a-z，A-Z，0-9和“_”这4种类型的字符，等价于[a-zA-Z0-9_]，\n",
    "# +表示匹配前面的表达式1次或者多次。因此\\w+表示匹配上述4种类型的字符1次或多次。\n",
    "pattern = r\"\\w+\"\n",
    "print(re.findall(pattern, sentence))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9c18dada",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Did', 'you', 'spend', '$3', '.4', 'on', 'arxiv', '.org', 'for', 'your', 'pre', '-print', '?', 'No', ',', 'it', \"'s\", 'free', '!', 'It', \"'s\", '.', '.', '.']\n"
     ]
    }
   ],
   "source": [
    "# 可以在正则表达式中使用\\S来表示除了空格以外的所有字符（\\s在正则表达式中表示空格字符，\\S则相应的表示\\s的补集）\n",
    "# |表示或运算，*表示匹配前面的表达式0次或多次，\\S\\w* 表示先匹配除了空格以外的1个字符，后面可以包含0个或多个\\w字符。\n",
    "pattern = r\"\\w+|\\S\\w*\"\n",
    "print(re.findall(pattern, sentence))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0a0d5c4c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Did', 'you', 'spend', '3', '4', 'on', 'arxiv', 'org', 'for', 'your', 'pre-print', 'No', \"it's\", 'free', \"It's\"]\n"
     ]
    }
   ],
   "source": [
    "# -表示匹配连字符-，(?:[-']\\w+)*表示匹配0次或多次括号内的模式。(?:...)表示匹配括号内的模式，\n",
    "# 可以和+/*等符号连用。其中?:表示不保存匹配到的括号中的内容，是re代码库中的特殊标准要求的部分。\n",
    "pattern = r\"\\w+(?:[-']\\w+)*\"\n",
    "print(re.findall(pattern, sentence))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c0394ce8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Did', 'you', 'spend', '$3', '.4', 'on', 'arxiv', '.org', 'for', 'your', 'pre-print', '?', 'No', ',', \"it's\", 'free', '!', \"It's\", '.', '.', '.']\n"
     ]
    }
   ],
   "source": [
    "pattern = r\"\\w+(?:[-']\\w+)*|\\S\\w*\"\n",
    "print(re.findall(pattern, sentence))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ea6d0b51",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Did', 'you', 'spend', '$3', '.4', 'on', 'arxiv.org', 'for', 'your', 'pre-print', '?', 'No', ',', \"it's\", 'free', '!', \"It's\", '.', '.', '.']\n"
     ]
    }
   ],
   "source": [
    "#新的匹配模式\n",
    "new_pattern = r\"(?:\\w+\\.)+\\w+(?:\\.)*\"\n",
    "pattern = new_pattern +r\"|\"+pattern\n",
    "print(re.findall(pattern, sentence))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "22563270",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Did', 'you', 'spend', '$3.4', 'on', 'arxiv.org', 'for', 'your', 'pre-print', '?', 'No', ',', \"it's\", 'free', '!', \"It's\", '.', '.', '.']\n"
     ]
    }
   ],
   "source": [
    "#新的匹配pattern，匹配价格符号\n",
    "new_pattern2 = r\"\\$?\\d+(?:\\.\\d+)?%?\"\n",
    "pattern = new_pattern2 +r\"|\" + new_pattern +r\"|\"+pattern\n",
    "print(re.findall(pattern, sentence))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "557c874d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Did', 'you', 'spend', '$3.4', 'on', 'arxiv.org', 'for', 'your', 'pre-print', '?', 'No', ',', \"it's\", 'free', '!', \"It's\", '...']\n"
     ]
    }
   ],
   "source": [
    "#新的匹配pattern，匹配价格符号\n",
    "new_pattern3 = r\"\\.\\.\\.\" \n",
    "pattern = new_pattern3 +r\"|\" + new_pattern2 +r\"|\" +\\\n",
    "    new_pattern +r\"|\"+pattern\n",
    "print(re.findall(pattern, sentence))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a17c005e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Did', 'you', 'spend', '$3.4', 'on', 'arxiv.org', 'for', 'your', 'pre-print', '?', 'No', ',', \"it's\", 'free', '!', \"It's\", '...']\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import nltk\n",
    "#引入NLTK分词器\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.tokenize import regexp_tokenize\n",
    "\n",
    "tokens = regexp_tokenize(sentence,pattern)\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7e1f552a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "语料：\n",
      "5 ['n', 'a', 'n', '_']\n",
      "2 ['n', 'a', 'n', 'j', 'i', 'n', 'g', '_']\n",
      "6 ['b', 'e', 'i', 'j', 'i', 'n', 'g', '_']\n",
      "3 ['d', 'o', 'n', 'g', 'b', 'e', 'i', '_']\n",
      "2 ['b', 'e', 'i', '_']\n",
      "词表：['_', 'a', 'b', 'd', 'e', 'g', 'i', 'j', 'n', 'o']\n"
     ]
    }
   ],
   "source": [
    "corpus = \"nan nan nan nan nan nanjing nanjing beijing beijing \"+\\\n",
    "    \"beijing beijing beijing beijing dongbei dongbei dongbei bei bei\"\n",
    "tokens = corpus.split(' ')\n",
    "\n",
    "#构建基于字符的初始词表\n",
    "vocabulary = set(corpus) \n",
    "vocabulary.remove(' ')\n",
    "vocabulary.add('_')\n",
    "vocabulary = sorted(list(vocabulary))\n",
    "\n",
    "#根据语料构建词表\n",
    "corpus_dict = {}\n",
    "for token in tokens:\n",
    "    key = token+'_'\n",
    "    if key not in corpus_dict:\n",
    "        corpus_dict[key] = {\"split\": list(key), \"count\": 0}\n",
    "    corpus_dict[key]['count'] += 1\n",
    "\n",
    "print(f\"语料：\")\n",
    "for key in corpus_dict:\n",
    "    print(corpus_dict[key]['count'], corpus_dict[key]['split'])\n",
    "print(f\"词表：{vocabulary}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "56056fdd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "第1次迭代\n",
      "当前最常出现的前5个符号组合：[('ng', 11), ('be', 11), ('ei', 11), ('ji', 8), ('in', 8)]\n",
      "本次迭代组合的符号为：ng\n",
      "\n",
      "迭代后的语料为：\n",
      "5 ['n', 'a', 'n', '_']\n",
      "2 ['n', 'a', 'n', 'j', 'i', 'ng', '_']\n",
      "6 ['b', 'e', 'i', 'j', 'i', 'ng', '_']\n",
      "3 ['d', 'o', 'ng', 'b', 'e', 'i', '_']\n",
      "2 ['b', 'e', 'i', '_']\n",
      "词表：['_', 'a', 'b', 'd', 'e', 'g', 'i', 'j', 'n', 'o', 'ng']\n",
      "\n",
      "-------------------------------------\n",
      "第2次迭代\n",
      "当前最常出现的前5个符号组合：[('be', 11), ('ei', 11), ('ji', 8), ('ing', 8), ('ng_', 8)]\n",
      "本次迭代组合的符号为：be\n",
      "\n",
      "迭代后的语料为：\n",
      "5 ['n', 'a', 'n', '_']\n",
      "2 ['n', 'a', 'n', 'j', 'i', 'ng', '_']\n",
      "6 ['be', 'i', 'j', 'i', 'ng', '_']\n",
      "3 ['d', 'o', 'ng', 'be', 'i', '_']\n",
      "2 ['be', 'i', '_']\n",
      "词表：['_', 'a', 'b', 'd', 'e', 'g', 'i', 'j', 'n', 'o', 'ng', 'be']\n",
      "\n",
      "-------------------------------------\n",
      "第3次迭代\n",
      "当前最常出现的前5个符号组合：[('bei', 11), ('ji', 8), ('ing', 8), ('ng_', 8), ('na', 7)]\n",
      "本次迭代组合的符号为：bei\n",
      "\n",
      "迭代后的语料为：\n",
      "5 ['n', 'a', 'n', '_']\n",
      "2 ['n', 'a', 'n', 'j', 'i', 'ng', '_']\n",
      "6 ['bei', 'j', 'i', 'ng', '_']\n",
      "3 ['d', 'o', 'ng', 'bei', '_']\n",
      "2 ['bei', '_']\n",
      "词表：['_', 'a', 'b', 'd', 'e', 'g', 'i', 'j', 'n', 'o', 'ng', 'be', 'bei']\n",
      "\n",
      "-------------------------------------\n",
      "第9次迭代\n",
      "当前最常出现的前5个符号组合：[('beijing_', 6), ('nan_', 5), ('bei_', 5), ('do', 3), ('ong', 3)]\n",
      "本次迭代组合的符号为：beijing_\n",
      "\n",
      "迭代后的语料为：\n",
      "5 ['nan', '_']\n",
      "2 ['nan', 'jing_']\n",
      "6 ['beijing_']\n",
      "3 ['d', 'o', 'ng', 'bei', '_']\n",
      "2 ['bei', '_']\n",
      "词表：['_', 'a', 'b', 'd', 'e', 'g', 'i', 'j', 'n', 'o', 'ng', 'be', 'bei', 'ji', 'jing', 'jing_', 'na', 'nan', 'beijing_']\n",
      "\n",
      "-------------------------------------\n"
     ]
    }
   ],
   "source": [
    "for step in range(9):\n",
    "    # 如果想要将每一步的结果都输出，请读者自行将max_print_step改成999\n",
    "    max_print_step = 3\n",
    "    if step < max_print_step or step == 8: \n",
    "        print(f\"第{step+1}次迭代\")\n",
    "    split_dict = {}\n",
    "    for key in corpus_dict:\n",
    "        splits = corpus_dict[key]['split']\n",
    "        # 遍历所有符号进行统计\n",
    "        for i in range(len(splits)-1):\n",
    "            # 组合两个符号作为新的符号\n",
    "            current_group = splits[i]+splits[i+1]\n",
    "            if current_group not in split_dict:\n",
    "                split_dict[current_group] = 0\n",
    "            split_dict[current_group] += corpus_dict[key]['count']\n",
    "\n",
    "    group_hist=[(k, v) for k, v in sorted(split_dict.items(), \\\n",
    "        key=lambda item: item[1],reverse=True)]\n",
    "    if step < max_print_step or step == 8:\n",
    "        print(f\"当前最常出现的前5个符号组合：{group_hist[:5]}\")\n",
    "    \n",
    "    merge_key = group_hist[0][0]\n",
    "    if step < max_print_step or step == 8:\n",
    "        print(f\"本次迭代组合的符号为：{merge_key}\")\n",
    "    for key in corpus_dict:\n",
    "        if merge_key in key:\n",
    "            new_splits = []\n",
    "            splits = corpus_dict[key]['split']\n",
    "            i = 0\n",
    "            while i < len(splits):\n",
    "                if i+1>=len(splits):\n",
    "                    new_splits.append(splits[i])\n",
    "                    i+=1\n",
    "                    continue\n",
    "                if merge_key == splits[i]+splits[i+1]:\n",
    "                    new_splits.append(merge_key)\n",
    "                    i+=2\n",
    "                else:\n",
    "                    new_splits.append(splits[i])\n",
    "                    i+=1\n",
    "            corpus_dict[key]['split']=new_splits\n",
    "            \n",
    "    vocabulary.append(merge_key)\n",
    "    if step < max_print_step or step == 8:\n",
    "        print()\n",
    "        print(f\"迭代后的语料为：\")\n",
    "        for key in corpus_dict:\n",
    "            print(corpus_dict[key]['count'], corpus_dict[key]['split'])\n",
    "        print(f\"词表：{vocabulary}\")\n",
    "        print()\n",
    "        print('-------------------------------------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "10a95f7d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "输入语句：nanjing beijing\n",
      "分词结果：['nan', 'jing_', 'beijing_']\n"
     ]
    }
   ],
   "source": [
    "ordered_vocabulary = {key: x for x, key in enumerate(vocabulary)}\n",
    "sentence = \"nanjing beijing\"\n",
    "print(f\"输入语句：{sentence}\")\n",
    "tokens = sentence.split(' ')\n",
    "tokenized_string = []\n",
    "for token in tokens:\n",
    "    key = token+'_'\n",
    "    splits = list(key)\n",
    "    #用于在没有更新的时候跳出\n",
    "    flag = 1\n",
    "    while flag:\n",
    "        flag = 0\n",
    "        split_dict = {}\n",
    "        #遍历所有符号进行统计\n",
    "        for i in range(len(splits)-1): \n",
    "            #组合两个符号作为新的符号\n",
    "            current_group = splits[i]+splits[i+1] \n",
    "            if current_group not in ordered_vocabulary:\n",
    "                continue\n",
    "            if current_group not in split_dict:\n",
    "                #判断当前组合是否在词表里，如果是的话加入split_dict\n",
    "                split_dict[current_group] = ordered_vocabulary[current_group] \n",
    "                flag = 1\n",
    "        if not flag:\n",
    "            continue\n",
    "            \n",
    "        #对每个组合进行优先级的排序（此处为从小到大）\n",
    "        group_hist=[(k, v) for k, v in sorted(split_dict.items(),\\\n",
    "            key=lambda item: item[1])] \n",
    "        #优先级最高的组合\n",
    "        merge_key = group_hist[0][0] \n",
    "        new_splits = []\n",
    "        i = 0\n",
    "         # 根据优先级最高的组合产生新的分词\n",
    "        while i < len(splits):\n",
    "            if i+1>=len(splits):\n",
    "                new_splits.append(splits[i])\n",
    "                i+=1\n",
    "                continue\n",
    "            if merge_key == splits[i]+splits[i+1]:\n",
    "                new_splits.append(merge_key)\n",
    "                i+=2\n",
    "            else:\n",
    "                new_splits.append(splits[i])\n",
    "                i+=1\n",
    "        splits=new_splits\n",
    "    tokenized_string+=splits\n",
    "\n",
    "print(f\"分词结果：{tokenized_string}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "6378257a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "let's study hands-on-nlp\n"
     ]
    }
   ],
   "source": [
    "# Case Folding\n",
    "sentence = \"Let's study Hands-on-NLP\"\n",
    "print(sentence.lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4f2181b1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "词目还原前：['Two', 'dogs', 'are', 'chasing', 'three', 'cats']\n",
      "词目还原后：['Two', 'dog', 'be', 'chase', 'three', 'cat']\n"
     ]
    }
   ],
   "source": [
    "# 构建词典\n",
    "lemma_dict = {'am': 'be','is': 'be','are': 'be','cats': 'cat',\\\n",
    "    \"cats'\": 'cat',\"cat's\": 'cat','dogs': 'dog',\"dogs'\": 'dog',\\\n",
    "    \"dog's\": 'dog', 'chasing': \"chase\"}\n",
    "\n",
    "sentence = \"Two dogs are chasing three cats\"\n",
    "words = sentence.split(' ')\n",
    "print(f'词目还原前：{words}')\n",
    "lemmatized_words = []\n",
    "for word in words:\n",
    "    if word in lemma_dict:\n",
    "        lemmatized_words.append(lemma_dict[word])\n",
    "    else:\n",
    "        lemmatized_words.append(word)\n",
    "\n",
    "print(f'词目还原后：{lemmatized_words}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6f36fed1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Error loading punkt: <urlopen error [Errno 11004]\n",
      "[nltk_data]     getaddrinfo failed>\n",
      "[nltk_data] Error loading wordnet: <urlopen error [Errno 11004]\n",
      "[nltk_data]     getaddrinfo failed>\n"
     ]
    },
    {
     "ename": "LookupError",
     "evalue": "\n**********************************************************************\n  Resource \u001b[93mpunkt\u001b[0m not found.\n  Please use the NLTK Downloader to obtain the resource:\n\n  \u001b[31m>>> import nltk\n  >>> nltk.download('punkt')\n  \u001b[0m\n  For more information see: https://www.nltk.org/data.html\n\n  Attempted to load \u001b[93mtokenizers/punkt/english.pickle\u001b[0m\n\n  Searched in:\n    - 'C:\\\\Users\\\\user/nltk_data'\n    - 'D:\\\\Program Files\\\\Anaconda3\\\\nltk_data'\n    - 'D:\\\\Program Files\\\\Anaconda3\\\\share\\\\nltk_data'\n    - 'D:\\\\Program Files\\\\Anaconda3\\\\lib\\\\nltk_data'\n    - 'C:\\\\Users\\\\user\\\\AppData\\\\Roaming\\\\nltk_data'\n    - 'C:\\\\nltk_data'\n    - 'D:\\\\nltk_data'\n    - 'E:\\\\nltk_data'\n    - ''\n**********************************************************************\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mLookupError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[23], line 15\u001b[0m\n\u001b[0;32m     12\u001b[0m lemmatizer \u001b[38;5;241m=\u001b[39m WordNetLemmatizer()\n\u001b[0;32m     13\u001b[0m sentence \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTwo dogs are chasing three cats\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m---> 15\u001b[0m words \u001b[38;5;241m=\u001b[39m word_tokenize(sentence)\n\u001b[0;32m     16\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m词目还原前：\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mwords\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m     17\u001b[0m lemmatized_words \u001b[38;5;241m=\u001b[39m []\n",
      "File \u001b[1;32mD:\\Program Files\\Anaconda3\\Lib\\site-packages\\nltk\\tokenize\\__init__.py:129\u001b[0m, in \u001b[0;36mword_tokenize\u001b[1;34m(text, language, preserve_line)\u001b[0m\n\u001b[0;32m    114\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mword_tokenize\u001b[39m(text, language\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124menglish\u001b[39m\u001b[38;5;124m\"\u001b[39m, preserve_line\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m):\n\u001b[0;32m    115\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m    116\u001b[0m \u001b[38;5;124;03m    Return a tokenized copy of *text*,\u001b[39;00m\n\u001b[0;32m    117\u001b[0m \u001b[38;5;124;03m    using NLTK's recommended word tokenizer\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    127\u001b[0m \u001b[38;5;124;03m    :type preserve_line: bool\u001b[39;00m\n\u001b[0;32m    128\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> 129\u001b[0m     sentences \u001b[38;5;241m=\u001b[39m [text] \u001b[38;5;28;01mif\u001b[39;00m preserve_line \u001b[38;5;28;01melse\u001b[39;00m sent_tokenize(text, language)\n\u001b[0;32m    130\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m [\n\u001b[0;32m    131\u001b[0m         token \u001b[38;5;28;01mfor\u001b[39;00m sent \u001b[38;5;129;01min\u001b[39;00m sentences \u001b[38;5;28;01mfor\u001b[39;00m token \u001b[38;5;129;01min\u001b[39;00m _treebank_word_tokenizer\u001b[38;5;241m.\u001b[39mtokenize(sent)\n\u001b[0;32m    132\u001b[0m     ]\n",
      "File \u001b[1;32mD:\\Program Files\\Anaconda3\\Lib\\site-packages\\nltk\\tokenize\\__init__.py:106\u001b[0m, in \u001b[0;36msent_tokenize\u001b[1;34m(text, language)\u001b[0m\n\u001b[0;32m     96\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21msent_tokenize\u001b[39m(text, language\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124menglish\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m     97\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m     98\u001b[0m \u001b[38;5;124;03m    Return a sentence-tokenized copy of *text*,\u001b[39;00m\n\u001b[0;32m     99\u001b[0m \u001b[38;5;124;03m    using NLTK's recommended sentence tokenizer\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    104\u001b[0m \u001b[38;5;124;03m    :param language: the model name in the Punkt corpus\u001b[39;00m\n\u001b[0;32m    105\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> 106\u001b[0m     tokenizer \u001b[38;5;241m=\u001b[39m load(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtokenizers/punkt/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mlanguage\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.pickle\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m    107\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m tokenizer\u001b[38;5;241m.\u001b[39mtokenize(text)\n",
      "File \u001b[1;32mD:\\Program Files\\Anaconda3\\Lib\\site-packages\\nltk\\data.py:750\u001b[0m, in \u001b[0;36mload\u001b[1;34m(resource_url, format, cache, verbose, logic_parser, fstruct_reader, encoding)\u001b[0m\n\u001b[0;32m    747\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m<<Loading \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresource_url\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m>>\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m    749\u001b[0m \u001b[38;5;66;03m# Load the resource.\u001b[39;00m\n\u001b[1;32m--> 750\u001b[0m opened_resource \u001b[38;5;241m=\u001b[39m _open(resource_url)\n\u001b[0;32m    752\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mformat\u001b[39m \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mraw\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m    753\u001b[0m     resource_val \u001b[38;5;241m=\u001b[39m opened_resource\u001b[38;5;241m.\u001b[39mread()\n",
      "File \u001b[1;32mD:\\Program Files\\Anaconda3\\Lib\\site-packages\\nltk\\data.py:876\u001b[0m, in \u001b[0;36m_open\u001b[1;34m(resource_url)\u001b[0m\n\u001b[0;32m    873\u001b[0m protocol, path_ \u001b[38;5;241m=\u001b[39m split_resource_url(resource_url)\n\u001b[0;32m    875\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m protocol \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m protocol\u001b[38;5;241m.\u001b[39mlower() \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnltk\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m--> 876\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m find(path_, path \u001b[38;5;241m+\u001b[39m [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m])\u001b[38;5;241m.\u001b[39mopen()\n\u001b[0;32m    877\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m protocol\u001b[38;5;241m.\u001b[39mlower() \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfile\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m    878\u001b[0m     \u001b[38;5;66;03m# urllib might not use mode='rb', so handle this one ourselves:\u001b[39;00m\n\u001b[0;32m    879\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m find(path_, [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m])\u001b[38;5;241m.\u001b[39mopen()\n",
      "File \u001b[1;32mD:\\Program Files\\Anaconda3\\Lib\\site-packages\\nltk\\data.py:583\u001b[0m, in \u001b[0;36mfind\u001b[1;34m(resource_name, paths)\u001b[0m\n\u001b[0;32m    581\u001b[0m sep \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m*\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m*\u001b[39m \u001b[38;5;241m70\u001b[39m\n\u001b[0;32m    582\u001b[0m resource_not_found \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00msep\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00mmsg\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00msep\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m--> 583\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mLookupError\u001b[39;00m(resource_not_found)\n",
      "\u001b[1;31mLookupError\u001b[0m: \n**********************************************************************\n  Resource \u001b[93mpunkt\u001b[0m not found.\n  Please use the NLTK Downloader to obtain the resource:\n\n  \u001b[31m>>> import nltk\n  >>> nltk.download('punkt')\n  \u001b[0m\n  For more information see: https://www.nltk.org/data.html\n\n  Attempted to load \u001b[93mtokenizers/punkt/english.pickle\u001b[0m\n\n  Searched in:\n    - 'C:\\\\Users\\\\user/nltk_data'\n    - 'D:\\\\Program Files\\\\Anaconda3\\\\nltk_data'\n    - 'D:\\\\Program Files\\\\Anaconda3\\\\share\\\\nltk_data'\n    - 'D:\\\\Program Files\\\\Anaconda3\\\\lib\\\\nltk_data'\n    - 'C:\\\\Users\\\\user\\\\AppData\\\\Roaming\\\\nltk_data'\n    - 'C:\\\\nltk_data'\n    - 'D:\\\\nltk_data'\n    - 'E:\\\\nltk_data'\n    - ''\n**********************************************************************\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "#引入nltk分词器、lemmatizer，引入wordnet还原动词\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.corpus import wordnet\n",
    "\n",
    "#下载分词包、wordnet包\n",
    "nltk.download('punkt', quiet=True)\n",
    "nltk.download('wordnet', quiet=True)\n",
    "\n",
    "\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "sentence = \"Two dogs are chasing three cats\"\n",
    "\n",
    "words = word_tokenize(sentence)\n",
    "print(f'词目还原前：{words}')\n",
    "lemmatized_words = []\n",
    "for word in words:\n",
    "    lemmatized_words.append(lemmatizer.lemmatize(word, wordnet.VERB))\n",
    "\n",
    "print(f'词目还原后：{lemmatized_words}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "50e6f018",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "分句结果：\n",
      "['Did', 'you', 'spend', '$3.4', 'on', 'arxiv.org', 'for', 'your', 'pre-print', '?']\n",
      "['No', ',', \"it's\", 'free', '!']\n",
      "[\"It's\", '...']\n"
     ]
    }
   ],
   "source": [
    "sentence_spliter = set([\".\",\"?\",'!','...'])\n",
    "sentence = \"Did you spend $3.4 on arxiv.org for your pre-print? \" + \\\n",
    "    \"No, it's free! It's ...\"\n",
    "\n",
    "tokens = regexp_tokenize(sentence,pattern)\n",
    "\n",
    "sentences = []\n",
    "boundary = [0]\n",
    "for token_id, token in enumerate(tokens):\n",
    "    # 判断句子边界\n",
    "    if token in sentence_spliter:\n",
    "        #如果是句子边界，则把分句结果加入进去\n",
    "        sentences.append(tokens[boundary[-1]:token_id+1]) \n",
    "        #将下一句句子起始位置加入boundary\n",
    "        boundary.append(token_id+1) \n",
    "\n",
    "if boundary[-1]!=len(tokens):\n",
    "    sentences.append(tokens[boundary[-1]:])\n",
    "\n",
    "print(f\"分句结果：\")\n",
    "for seg_sentence in sentences:\n",
    "    print(seg_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fea11e61",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
